{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 过滤数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from pathlib import Path\n",
    "import os \n",
    "from PIL import Image\n",
    "import shutil\n",
    "from glob import glob\n",
    "from tqdm import tqdm \n",
    "from joblib import Parallel, delayed\n",
    "from typing import Union\n",
    "from torchvision.io import ImageReadMode, read_image\n",
    "from transformers import ChineseCLIPProcessor, ChineseCLIPModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bigdata/image_data\\\\test-0.jpg', 'bigdata/image_data\\\\test-1.jpg']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_image_path_list = glob(\"bigdata/image_data/*\")#[:20]\n",
    "all_image_path_list[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_or_path = \"OFA-Sys/chinese-clip-vit-large-patch14\"\n",
    "processor = ChineseCLIPProcessor.from_pretrained(model_name_or_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 20 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.0191s.) Setting batch_size=2.\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done  21 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  50 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=-1)]: Done  76 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=-1)]: Done 106 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1950s.) Setting batch_size=4.\n",
      "[Parallel(n_jobs=-1)]: Done 136 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Done 170 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=-1)]: Done 204 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=-1)]: Done 244 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=-1)]: Done 320 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=-1)]: Done 404 tasks      | elapsed:    1.3s\n",
      "[Parallel(n_jobs=-1)]: Done 488 tasks      | elapsed:    1.6s\n",
      "[Parallel(n_jobs=-1)]: Done 580 tasks      | elapsed:    2.0s\n",
      "[Parallel(n_jobs=-1)]: Done 672 tasks      | elapsed:    2.3s\n",
      "[Parallel(n_jobs=-1)]: Done 772 tasks      | elapsed:    2.8s\n",
      "[Parallel(n_jobs=-1)]: Done 872 tasks      | elapsed:    3.2s\n",
      "[Parallel(n_jobs=-1)]: Done 980 tasks      | elapsed:    3.5s\n",
      "[Parallel(n_jobs=-1)]: Done 1088 tasks      | elapsed:    3.9s\n",
      "[Parallel(n_jobs=-1)]: Done 1204 tasks      | elapsed:    4.3s\n",
      "[Parallel(n_jobs=-1)]: Done 1320 tasks      | elapsed:    4.6s\n",
      "[Parallel(n_jobs=-1)]: Done 1444 tasks      | elapsed:    5.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1568 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=-1)]: Done 1700 tasks      | elapsed:    5.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1832 tasks      | elapsed:    6.3s\n",
      "[Parallel(n_jobs=-1)]: Done 1972 tasks      | elapsed:    6.8s\n",
      "[Parallel(n_jobs=-1)]: Done 2112 tasks      | elapsed:    7.2s\n",
      "[Parallel(n_jobs=-1)]: Done 2260 tasks      | elapsed:    7.7s\n",
      "[Parallel(n_jobs=-1)]: Done 2408 tasks      | elapsed:    8.2s\n",
      "[Parallel(n_jobs=-1)]: Done 2564 tasks      | elapsed:    8.8s\n",
      "[Parallel(n_jobs=-1)]: Done 2720 tasks      | elapsed:    9.4s\n",
      "[Parallel(n_jobs=-1)]: Done 2884 tasks      | elapsed:    9.9s\n",
      "[Parallel(n_jobs=-1)]: Done 3048 tasks      | elapsed:   10.4s\n",
      "[Parallel(n_jobs=-1)]: Done 3220 tasks      | elapsed:   10.9s\n",
      "[Parallel(n_jobs=-1)]: Done 3392 tasks      | elapsed:   11.6s\n",
      "[Parallel(n_jobs=-1)]: Done 3572 tasks      | elapsed:   12.1s\n",
      "[Parallel(n_jobs=-1)]: Done 3752 tasks      | elapsed:   12.7s\n",
      "[Parallel(n_jobs=-1)]: Done 3940 tasks      | elapsed:   13.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4128 tasks      | elapsed:   13.9s\n",
      "[Parallel(n_jobs=-1)]: Done 4324 tasks      | elapsed:   14.4s\n",
      "[Parallel(n_jobs=-1)]: Done 4520 tasks      | elapsed:   15.0s\n",
      "[Parallel(n_jobs=-1)]: Done 4724 tasks      | elapsed:   15.8s\n",
      "[Parallel(n_jobs=-1)]: Done 4928 tasks      | elapsed:   16.4s\n",
      "[Parallel(n_jobs=-1)]: Done 5140 tasks      | elapsed:   17.2s\n",
      "[Parallel(n_jobs=-1)]: Done 5352 tasks      | elapsed:   18.0s\n",
      "[Parallel(n_jobs=-1)]: Done 5572 tasks      | elapsed:   18.6s\n",
      "[Parallel(n_jobs=-1)]: Done 5792 tasks      | elapsed:   19.4s\n",
      "[Parallel(n_jobs=-1)]: Done 6020 tasks      | elapsed:   20.0s\n",
      "[Parallel(n_jobs=-1)]: Done 6248 tasks      | elapsed:   20.7s\n",
      "[Parallel(n_jobs=-1)]: Done 6484 tasks      | elapsed:   21.6s\n",
      "[Parallel(n_jobs=-1)]: Done 6720 tasks      | elapsed:   22.5s\n",
      "[Parallel(n_jobs=-1)]: Done 6964 tasks      | elapsed:   23.4s\n",
      "[Parallel(n_jobs=-1)]: Done 7208 tasks      | elapsed:   24.3s\n",
      "[Parallel(n_jobs=-1)]: Done 7460 tasks      | elapsed:   25.3s\n",
      "[Parallel(n_jobs=-1)]: Done 7712 tasks      | elapsed:   26.3s\n",
      "[Parallel(n_jobs=-1)]: Done 7972 tasks      | elapsed:   27.3s\n",
      "[Parallel(n_jobs=-1)]: Done 8232 tasks      | elapsed:   28.3s\n",
      "[Parallel(n_jobs=-1)]: Done 8500 tasks      | elapsed:   29.3s\n",
      "[Parallel(n_jobs=-1)]: Done 8768 tasks      | elapsed:   30.1s\n",
      "[Parallel(n_jobs=-1)]: Done 9044 tasks      | elapsed:   31.2s\n",
      "[Parallel(n_jobs=-1)]: Done 9320 tasks      | elapsed:   32.2s\n",
      "[Parallel(n_jobs=-1)]: Done 9604 tasks      | elapsed:   33.1s\n",
      "[Parallel(n_jobs=-1)]: Done 9888 tasks      | elapsed:   34.1s\n",
      "[Parallel(n_jobs=-1)]: Done 10180 tasks      | elapsed:   35.1s\n",
      "[Parallel(n_jobs=-1)]: Done 10472 tasks      | elapsed:   36.1s\n",
      "[Parallel(n_jobs=-1)]: Done 10772 tasks      | elapsed:   37.0s\n",
      "[Parallel(n_jobs=-1)]: Done 11072 tasks      | elapsed:   37.9s\n",
      "[Parallel(n_jobs=-1)]: Done 11380 tasks      | elapsed:   39.0s\n",
      "[Parallel(n_jobs=-1)]: Done 11688 tasks      | elapsed:   39.9s\n",
      "[Parallel(n_jobs=-1)]: Done 12004 tasks      | elapsed:   40.9s\n",
      "[Parallel(n_jobs=-1)]: Done 12320 tasks      | elapsed:   41.9s\n",
      "[Parallel(n_jobs=-1)]: Done 12644 tasks      | elapsed:   42.9s\n",
      "[Parallel(n_jobs=-1)]: Done 12968 tasks      | elapsed:   44.0s\n",
      "[Parallel(n_jobs=-1)]: Done 13300 tasks      | elapsed:   45.1s\n",
      "[Parallel(n_jobs=-1)]: Done 13632 tasks      | elapsed:   46.1s\n",
      "[Parallel(n_jobs=-1)]: Done 13972 tasks      | elapsed:   47.0s\n",
      "[Parallel(n_jobs=-1)]: Done 14312 tasks      | elapsed:   48.1s\n",
      "[Parallel(n_jobs=-1)]: Done 14660 tasks      | elapsed:   49.2s\n",
      "[Parallel(n_jobs=-1)]: Done 15008 tasks      | elapsed:   50.2s\n",
      "[Parallel(n_jobs=-1)]: Done 15364 tasks      | elapsed:   51.3s\n",
      "[Parallel(n_jobs=-1)]: Done 15720 tasks      | elapsed:   52.3s\n",
      "[Parallel(n_jobs=-1)]: Done 16084 tasks      | elapsed:   53.4s\n",
      "[Parallel(n_jobs=-1)]: Done 16448 tasks      | elapsed:   54.4s\n",
      "[Parallel(n_jobs=-1)]: Done 16820 tasks      | elapsed:   55.6s\n",
      "[Parallel(n_jobs=-1)]: Done 17192 tasks      | elapsed:   56.8s\n",
      "[Parallel(n_jobs=-1)]: Done 17572 tasks      | elapsed:   57.9s\n",
      "[Parallel(n_jobs=-1)]: Done 17952 tasks      | elapsed:   59.2s\n",
      "[Parallel(n_jobs=-1)]: Done 18340 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 18728 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 19124 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 19520 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 19924 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 20328 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 20740 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 21152 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 21572 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 21992 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22420 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22848 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 23284 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 23720 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 24164 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 24608 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 25060 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 25512 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 25972 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 26432 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 26900 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 27368 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 27844 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 28320 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 28804 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 29288 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 29780 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 30272 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 30772 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 31272 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 31780 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 32288 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 32804 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 33320 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 33844 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 34368 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 34900 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 35432 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 35972 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 36512 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 37060 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 37608 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 38164 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 38720 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 39284 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 39848 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 40420 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 40992 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 41572 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 42152 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 42740 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 43328 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 43924 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 44520 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 45124 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 45728 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 46340 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 46952 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 47572 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 48192 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 48820 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 49448 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 50084 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 50720 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 51364 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 52008 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 52660 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 53312 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 53972 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 54632 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 55300 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 55968 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 56644 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 57320 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 58004 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 58688 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 59380 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 60072 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 60772 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 61472 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 62180 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 62888 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 63604 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 64320 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 65044 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 65768 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 66500 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 67232 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 67972 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 68712 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 69460 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 70208 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 70964 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 71720 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 72484 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 73248 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 74020 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 74792 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 75572 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 76352 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 77140 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 77928 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 78724 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 79520 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 80324 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 81128 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 81940 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 82752 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 83572 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 84392 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 85220 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 86048 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 86884 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 87720 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 88564 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 89408 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 90260 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 91112 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 91972 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 92832 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 93700 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 94568 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=-1)]: Done 95444 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=-1)]: Done 96320 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 97204 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 98088 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 98980 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 99872 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 100772 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 101672 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 102580 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=-1)]: Done 103488 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=-1)]: Done 104404 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 105320 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 106244 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 107168 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 108100 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1992s.) Setting batch_size=8.\n",
      "[Parallel(n_jobs=-1)]: Done 109744 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done 111624 tasks      | elapsed:  5.8min\n",
      "[Parallel(n_jobs=-1)]: Done 113504 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=-1)]: Done 115400 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 117296 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 119208 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done 121120 tasks      | elapsed:  6.3min\n",
      "[Parallel(n_jobs=-1)]: Done 123048 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=-1)]: Done 124976 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=-1)]: Done 126920 tasks      | elapsed:  6.6min\n",
      "[Parallel(n_jobs=-1)]: Done 128864 tasks      | elapsed:  6.7min\n",
      "[Parallel(n_jobs=-1)]: Done 130824 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 132784 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 134760 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 136736 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 138728 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 140720 tasks      | elapsed:  7.2min\n",
      "[Parallel(n_jobs=-1)]: Done 142728 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=-1)]: Done 144736 tasks      | elapsed:  7.4min\n",
      "[Parallel(n_jobs=-1)]: Done 146760 tasks      | elapsed:  7.5min\n",
      "[Parallel(n_jobs=-1)]: Done 148784 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 150824 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 152864 tasks      | elapsed:  7.8min\n",
      "[Parallel(n_jobs=-1)]: Done 154920 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=-1)]: Done 156976 tasks      | elapsed:  8.0min\n",
      "[Parallel(n_jobs=-1)]: Done 159048 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=-1)]: Done 161120 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 163208 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done 165296 tasks      | elapsed:  8.4min\n",
      "[Parallel(n_jobs=-1)]: Done 167400 tasks      | elapsed:  8.5min\n",
      "[Parallel(n_jobs=-1)]: Done 169504 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 171624 tasks      | elapsed:  8.7min\n",
      "[Parallel(n_jobs=-1)]: Done 173744 tasks      | elapsed:  8.8min\n",
      "[Parallel(n_jobs=-1)]: Done 175880 tasks      | elapsed:  8.9min\n",
      "[Parallel(n_jobs=-1)]: Done 178016 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=-1)]: Done 180168 tasks      | elapsed:  9.1min\n",
      "[Parallel(n_jobs=-1)]: Done 182320 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 184488 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done 186656 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done 188840 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=-1)]: Done 191024 tasks      | elapsed:  9.7min\n",
      "[Parallel(n_jobs=-1)]: Done 193224 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 195424 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 197640 tasks      | elapsed: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done 199856 tasks      | elapsed: 10.1min\n",
      "[Parallel(n_jobs=-1)]: Done 202088 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=-1)]: Done 204320 tasks      | elapsed: 10.3min\n",
      "[Parallel(n_jobs=-1)]: Done 206568 tasks      | elapsed: 10.4min\n",
      "[Parallel(n_jobs=-1)]: Done 208816 tasks      | elapsed: 10.5min\n",
      "[Parallel(n_jobs=-1)]: Done 211080 tasks      | elapsed: 10.6min\n",
      "[Parallel(n_jobs=-1)]: Done 213344 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done 215624 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 217904 tasks      | elapsed: 11.0min\n",
      "[Parallel(n_jobs=-1)]: Done 220200 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 222496 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=-1)]: Done 224808 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=-1)]: Done 227120 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done 229448 tasks      | elapsed: 11.5min\n",
      "[Parallel(n_jobs=-1)]: Done 231776 tasks      | elapsed: 11.6min\n",
      "[Parallel(n_jobs=-1)]: Done 234120 tasks      | elapsed: 11.8min\n",
      "[Parallel(n_jobs=-1)]: Done 236464 tasks      | elapsed: 11.9min\n",
      "[Parallel(n_jobs=-1)]: Done 238824 tasks      | elapsed: 12.0min\n",
      "[Parallel(n_jobs=-1)]: Done 241184 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=-1)]: Done 243560 tasks      | elapsed: 12.2min\n",
      "[Parallel(n_jobs=-1)]: Done 245936 tasks      | elapsed: 12.3min\n",
      "[Parallel(n_jobs=-1)]: Done 248328 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=-1)]: Done 250720 tasks      | elapsed: 12.6min\n",
      "[Parallel(n_jobs=-1)]: Done 253128 tasks      | elapsed: 12.7min\n",
      "[Parallel(n_jobs=-1)]: Done 255536 tasks      | elapsed: 12.8min\n",
      "[Parallel(n_jobs=-1)]: Done 257960 tasks      | elapsed: 12.9min\n",
      "[Parallel(n_jobs=-1)]: Done 260384 tasks      | elapsed: 13.0min\n",
      "[Parallel(n_jobs=-1)]: Done 262824 tasks      | elapsed: 13.2min\n",
      "[Parallel(n_jobs=-1)]: Done 265264 tasks      | elapsed: 13.3min\n",
      "[Parallel(n_jobs=-1)]: Done 267720 tasks      | elapsed: 13.4min\n",
      "[Parallel(n_jobs=-1)]: Done 270176 tasks      | elapsed: 13.5min\n",
      "[Parallel(n_jobs=-1)]: Done 272648 tasks      | elapsed: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done 275120 tasks      | elapsed: 13.8min\n",
      "[Parallel(n_jobs=-1)]: Done 277608 tasks      | elapsed: 13.9min\n",
      "[Parallel(n_jobs=-1)]: Done 280096 tasks      | elapsed: 14.0min\n",
      "[Parallel(n_jobs=-1)]: Done 282600 tasks      | elapsed: 14.1min\n",
      "[Parallel(n_jobs=-1)]: Done 285104 tasks      | elapsed: 14.3min\n",
      "[Parallel(n_jobs=-1)]: Done 287624 tasks      | elapsed: 14.4min\n",
      "[Parallel(n_jobs=-1)]: Done 290144 tasks      | elapsed: 14.5min\n",
      "[Parallel(n_jobs=-1)]: Done 292680 tasks      | elapsed: 14.6min\n",
      "[Parallel(n_jobs=-1)]: Done 295216 tasks      | elapsed: 14.7min\n",
      "[Parallel(n_jobs=-1)]: Done 297768 tasks      | elapsed: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done 300320 tasks      | elapsed: 15.0min\n",
      "[Parallel(n_jobs=-1)]: Batch computation too slow (2.1035s.) Setting batch_size=1.\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.2000s.) Setting batch_size=2.\n",
      "[Parallel(n_jobs=-1)]: Done 302461 tasks      | elapsed: 15.1min\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1986s.) Setting batch_size=4.\n",
      "[Parallel(n_jobs=-1)]: Done 303428 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 304720 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 306012 tasks      | elapsed: 15.3min\n",
      "[Parallel(n_jobs=-1)]: Done 307312 tasks      | elapsed: 15.4min\n",
      "[Parallel(n_jobs=-1)]: Done 308612 tasks      | elapsed: 15.4min\n",
      "[Parallel(n_jobs=-1)]: Done 309920 tasks      | elapsed: 15.5min\n",
      "[Parallel(n_jobs=-1)]: Done 311228 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 312544 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 313860 tasks      | elapsed: 15.7min\n",
      "[Parallel(n_jobs=-1)]: Done 315184 tasks      | elapsed: 15.8min\n",
      "[Parallel(n_jobs=-1)]: Done 316508 tasks      | elapsed: 15.9min\n",
      "[Parallel(n_jobs=-1)]: Done 317840 tasks      | elapsed: 15.9min\n",
      "[Parallel(n_jobs=-1)]: Done 319172 tasks      | elapsed: 16.0min\n",
      "[Parallel(n_jobs=-1)]: Done 320512 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 321852 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 323200 tasks      | elapsed: 16.2min\n",
      "[Parallel(n_jobs=-1)]: Done 324548 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 325904 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 327260 tasks      | elapsed: 16.4min\n",
      "[Parallel(n_jobs=-1)]: Done 328624 tasks      | elapsed: 16.5min\n",
      "[Parallel(n_jobs=-1)]: Done 329988 tasks      | elapsed: 16.6min\n",
      "[Parallel(n_jobs=-1)]: Done 331360 tasks      | elapsed: 16.6min\n",
      "[Parallel(n_jobs=-1)]: Done 332732 tasks      | elapsed: 16.7min\n",
      "[Parallel(n_jobs=-1)]: Done 334112 tasks      | elapsed: 16.8min\n",
      "[Parallel(n_jobs=-1)]: Done 335492 tasks      | elapsed: 16.8min\n",
      "[Parallel(n_jobs=-1)]: Done 336880 tasks      | elapsed: 16.9min\n",
      "[Parallel(n_jobs=-1)]: Done 338268 tasks      | elapsed: 17.0min\n",
      "[Parallel(n_jobs=-1)]: Done 339664 tasks      | elapsed: 17.1min\n",
      "[Parallel(n_jobs=-1)]: Done 341060 tasks      | elapsed: 17.1min\n",
      "[Parallel(n_jobs=-1)]: Done 342464 tasks      | elapsed: 17.2min\n",
      "[Parallel(n_jobs=-1)]: Done 343868 tasks      | elapsed: 17.3min\n",
      "[Parallel(n_jobs=-1)]: Done 345280 tasks      | elapsed: 17.4min\n",
      "[Parallel(n_jobs=-1)]: Done 346692 tasks      | elapsed: 17.4min\n",
      "[Parallel(n_jobs=-1)]: Done 348112 tasks      | elapsed: 17.5min\n",
      "[Parallel(n_jobs=-1)]: Done 349532 tasks      | elapsed: 17.6min\n",
      "[Parallel(n_jobs=-1)]: Done 350960 tasks      | elapsed: 17.7min\n",
      "[Parallel(n_jobs=-1)]: Done 352388 tasks      | elapsed: 17.7min\n",
      "[Parallel(n_jobs=-1)]: Done 353824 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 355260 tasks      | elapsed: 17.9min\n",
      "[Parallel(n_jobs=-1)]: Done 356704 tasks      | elapsed: 18.0min\n",
      "[Parallel(n_jobs=-1)]: Done 358148 tasks      | elapsed: 18.0min\n",
      "[Parallel(n_jobs=-1)]: Done 359600 tasks      | elapsed: 18.1min\n",
      "[Parallel(n_jobs=-1)]: Done 361052 tasks      | elapsed: 18.2min\n",
      "[Parallel(n_jobs=-1)]: Done 362512 tasks      | elapsed: 18.3min\n",
      "[Parallel(n_jobs=-1)]: Done 363972 tasks      | elapsed: 18.3min\n",
      "[Parallel(n_jobs=-1)]: Done 365440 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 366908 tasks      | elapsed: 18.5min\n",
      "[Parallel(n_jobs=-1)]: Done 368384 tasks      | elapsed: 18.6min\n",
      "[Parallel(n_jobs=-1)]: Done 369860 tasks      | elapsed: 18.6min\n",
      "[Parallel(n_jobs=-1)]: Done 371344 tasks      | elapsed: 18.7min\n",
      "[Parallel(n_jobs=-1)]: Done 372828 tasks      | elapsed: 18.8min\n",
      "[Parallel(n_jobs=-1)]: Done 374320 tasks      | elapsed: 18.9min\n",
      "[Parallel(n_jobs=-1)]: Done 375812 tasks      | elapsed: 18.9min\n",
      "[Parallel(n_jobs=-1)]: Done 377312 tasks      | elapsed: 19.0min\n",
      "[Parallel(n_jobs=-1)]: Done 378812 tasks      | elapsed: 19.1min\n",
      "[Parallel(n_jobs=-1)]: Done 380320 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 381828 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 383344 tasks      | elapsed: 19.3min\n",
      "[Parallel(n_jobs=-1)]: Done 384860 tasks      | elapsed: 19.4min\n",
      "[Parallel(n_jobs=-1)]: Done 386384 tasks      | elapsed: 19.5min\n",
      "[Parallel(n_jobs=-1)]: Done 387908 tasks      | elapsed: 19.5min\n",
      "[Parallel(n_jobs=-1)]: Done 389440 tasks      | elapsed: 19.6min\n",
      "[Parallel(n_jobs=-1)]: Done 390972 tasks      | elapsed: 19.7min\n",
      "[Parallel(n_jobs=-1)]: Done 392512 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 394052 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 395600 tasks      | elapsed: 19.9min\n",
      "[Parallel(n_jobs=-1)]: Done 397148 tasks      | elapsed: 20.0min\n",
      "[Parallel(n_jobs=-1)]: Done 398704 tasks      | elapsed: 20.1min\n",
      "[Parallel(n_jobs=-1)]: Done 400260 tasks      | elapsed: 20.2min\n",
      "[Parallel(n_jobs=-1)]: Done 401824 tasks      | elapsed: 20.2min\n",
      "[Parallel(n_jobs=-1)]: Done 403388 tasks      | elapsed: 20.3min\n",
      "[Parallel(n_jobs=-1)]: Done 404960 tasks      | elapsed: 20.4min\n",
      "[Parallel(n_jobs=-1)]: Done 406532 tasks      | elapsed: 20.5min\n",
      "[Parallel(n_jobs=-1)]: Done 408112 tasks      | elapsed: 20.5min\n",
      "[Parallel(n_jobs=-1)]: Done 409692 tasks      | elapsed: 20.6min\n",
      "[Parallel(n_jobs=-1)]: Done 411280 tasks      | elapsed: 20.7min\n",
      "[Parallel(n_jobs=-1)]: Done 412868 tasks      | elapsed: 20.8min\n",
      "[Parallel(n_jobs=-1)]: Done 414464 tasks      | elapsed: 20.9min\n",
      "[Parallel(n_jobs=-1)]: Done 416060 tasks      | elapsed: 21.0min\n",
      "[Parallel(n_jobs=-1)]: Done 417664 tasks      | elapsed: 21.0min\n",
      "[Parallel(n_jobs=-1)]: Done 419268 tasks      | elapsed: 21.1min\n",
      "[Parallel(n_jobs=-1)]: Done 420880 tasks      | elapsed: 21.2min\n",
      "[Parallel(n_jobs=-1)]: Done 422492 tasks      | elapsed: 21.3min\n",
      "[Parallel(n_jobs=-1)]: Done 424112 tasks      | elapsed: 21.4min\n",
      "[Parallel(n_jobs=-1)]: Done 425732 tasks      | elapsed: 21.5min\n",
      "[Parallel(n_jobs=-1)]: Done 427360 tasks      | elapsed: 21.6min\n",
      "[Parallel(n_jobs=-1)]: Done 428988 tasks      | elapsed: 21.6min\n",
      "[Parallel(n_jobs=-1)]: Done 430624 tasks      | elapsed: 21.7min\n",
      "[Parallel(n_jobs=-1)]: Done 432260 tasks      | elapsed: 21.8min\n",
      "[Parallel(n_jobs=-1)]: Done 433904 tasks      | elapsed: 21.9min\n",
      "[Parallel(n_jobs=-1)]: Done 435548 tasks      | elapsed: 22.0min\n",
      "[Parallel(n_jobs=-1)]: Done 437200 tasks      | elapsed: 22.1min\n",
      "[Parallel(n_jobs=-1)]: Done 438852 tasks      | elapsed: 22.2min\n",
      "[Parallel(n_jobs=-1)]: Done 440512 tasks      | elapsed: 22.2min\n",
      "[Parallel(n_jobs=-1)]: Done 442172 tasks      | elapsed: 22.3min\n",
      "[Parallel(n_jobs=-1)]: Done 443840 tasks      | elapsed: 22.4min\n",
      "[Parallel(n_jobs=-1)]: Done 445508 tasks      | elapsed: 22.5min\n",
      "[Parallel(n_jobs=-1)]: Done 447184 tasks      | elapsed: 22.6min\n",
      "[Parallel(n_jobs=-1)]: Done 448860 tasks      | elapsed: 22.7min\n",
      "[Parallel(n_jobs=-1)]: Done 450544 tasks      | elapsed: 22.8min\n",
      "[Parallel(n_jobs=-1)]: Done 452228 tasks      | elapsed: 22.9min\n",
      "[Parallel(n_jobs=-1)]: Done 453920 tasks      | elapsed: 22.9min\n",
      "[Parallel(n_jobs=-1)]: Done 455612 tasks      | elapsed: 23.0min\n",
      "[Parallel(n_jobs=-1)]: Done 457312 tasks      | elapsed: 23.1min\n",
      "[Parallel(n_jobs=-1)]: Done 459012 tasks      | elapsed: 23.2min\n",
      "[Parallel(n_jobs=-1)]: Done 460720 tasks      | elapsed: 23.3min\n",
      "[Parallel(n_jobs=-1)]: Done 462428 tasks      | elapsed: 23.4min\n",
      "[Parallel(n_jobs=-1)]: Done 464144 tasks      | elapsed: 23.5min\n",
      "[Parallel(n_jobs=-1)]: Done 465860 tasks      | elapsed: 23.6min\n",
      "[Parallel(n_jobs=-1)]: Done 467584 tasks      | elapsed: 23.7min\n",
      "[Parallel(n_jobs=-1)]: Done 469308 tasks      | elapsed: 23.8min\n",
      "[Parallel(n_jobs=-1)]: Done 471040 tasks      | elapsed: 23.8min\n",
      "[Parallel(n_jobs=-1)]: Done 472772 tasks      | elapsed: 23.9min\n",
      "[Parallel(n_jobs=-1)]: Done 474512 tasks      | elapsed: 24.0min\n",
      "[Parallel(n_jobs=-1)]: Done 476252 tasks      | elapsed: 24.1min\n",
      "[Parallel(n_jobs=-1)]: Done 478000 tasks      | elapsed: 24.2min\n",
      "[Parallel(n_jobs=-1)]: Done 479748 tasks      | elapsed: 24.3min\n",
      "[Parallel(n_jobs=-1)]: Done 481504 tasks      | elapsed: 24.4min\n",
      "[Parallel(n_jobs=-1)]: Done 483260 tasks      | elapsed: 24.5min\n",
      "[Parallel(n_jobs=-1)]: Done 485024 tasks      | elapsed: 24.6min\n",
      "[Parallel(n_jobs=-1)]: Done 486788 tasks      | elapsed: 24.6min\n",
      "[Parallel(n_jobs=-1)]: Done 488560 tasks      | elapsed: 24.7min\n",
      "[Parallel(n_jobs=-1)]: Done 490332 tasks      | elapsed: 24.8min\n",
      "[Parallel(n_jobs=-1)]: Done 492112 tasks      | elapsed: 24.9min\n",
      "[Parallel(n_jobs=-1)]: Done 493892 tasks      | elapsed: 25.0min\n",
      "[Parallel(n_jobs=-1)]: Done 495680 tasks      | elapsed: 25.1min\n",
      "[Parallel(n_jobs=-1)]: Done 497468 tasks      | elapsed: 25.2min\n",
      "[Parallel(n_jobs=-1)]: Done 499264 tasks      | elapsed: 25.3min\n",
      "[Parallel(n_jobs=-1)]: Done 501060 tasks      | elapsed: 25.4min\n",
      "[Parallel(n_jobs=-1)]: Done 502864 tasks      | elapsed: 25.5min\n",
      "[Parallel(n_jobs=-1)]: Done 504668 tasks      | elapsed: 25.6min\n",
      "[Parallel(n_jobs=-1)]: Done 506480 tasks      | elapsed: 25.7min\n",
      "[Parallel(n_jobs=-1)]: Done 508292 tasks      | elapsed: 25.8min\n",
      "[Parallel(n_jobs=-1)]: Done 510112 tasks      | elapsed: 25.9min\n",
      "[Parallel(n_jobs=-1)]: Done 511932 tasks      | elapsed: 26.0min\n",
      "[Parallel(n_jobs=-1)]: Done 513760 tasks      | elapsed: 26.1min\n",
      "[Parallel(n_jobs=-1)]: Done 515588 tasks      | elapsed: 26.1min\n",
      "[Parallel(n_jobs=-1)]: Done 517424 tasks      | elapsed: 26.2min\n",
      "[Parallel(n_jobs=-1)]: Done 519260 tasks      | elapsed: 26.3min\n",
      "[Parallel(n_jobs=-1)]: Done 521104 tasks      | elapsed: 26.4min\n",
      "[Parallel(n_jobs=-1)]: Done 522948 tasks      | elapsed: 26.5min\n",
      "[Parallel(n_jobs=-1)]: Done 524800 tasks      | elapsed: 26.6min\n",
      "[Parallel(n_jobs=-1)]: Done 526652 tasks      | elapsed: 26.7min\n",
      "[Parallel(n_jobs=-1)]: Done 528512 tasks      | elapsed: 26.8min\n",
      "[Parallel(n_jobs=-1)]: Done 530372 tasks      | elapsed: 26.9min\n",
      "[Parallel(n_jobs=-1)]: Done 532240 tasks      | elapsed: 27.0min\n",
      "[Parallel(n_jobs=-1)]: Done 534108 tasks      | elapsed: 27.1min\n",
      "[Parallel(n_jobs=-1)]: Done 535984 tasks      | elapsed: 27.2min\n",
      "[Parallel(n_jobs=-1)]: Done 537860 tasks      | elapsed: 27.3min\n",
      "[Parallel(n_jobs=-1)]: Done 539744 tasks      | elapsed: 27.4min\n",
      "[Parallel(n_jobs=-1)]: Done 541628 tasks      | elapsed: 27.5min\n",
      "[Parallel(n_jobs=-1)]: Done 543520 tasks      | elapsed: 27.6min\n",
      "[Parallel(n_jobs=-1)]: Done 545412 tasks      | elapsed: 27.7min\n",
      "[Parallel(n_jobs=-1)]: Done 547312 tasks      | elapsed: 27.8min\n",
      "[Parallel(n_jobs=-1)]: Done 549212 tasks      | elapsed: 27.9min\n",
      "[Parallel(n_jobs=-1)]: Done 551120 tasks      | elapsed: 28.0min\n",
      "[Parallel(n_jobs=-1)]: Done 553028 tasks      | elapsed: 28.1min\n",
      "[Parallel(n_jobs=-1)]: Done 554944 tasks      | elapsed: 28.2min\n",
      "[Parallel(n_jobs=-1)]: Done 556860 tasks      | elapsed: 28.3min\n",
      "[Parallel(n_jobs=-1)]: Done 558784 tasks      | elapsed: 28.4min\n",
      "[Parallel(n_jobs=-1)]: Done 560708 tasks      | elapsed: 28.5min\n",
      "[Parallel(n_jobs=-1)]: Done 562640 tasks      | elapsed: 28.6min\n",
      "[Parallel(n_jobs=-1)]: Done 564572 tasks      | elapsed: 28.7min\n",
      "[Parallel(n_jobs=-1)]: Done 566512 tasks      | elapsed: 28.8min\n",
      "[Parallel(n_jobs=-1)]: Done 568452 tasks      | elapsed: 28.9min\n",
      "[Parallel(n_jobs=-1)]: Done 570400 tasks      | elapsed: 29.0min\n",
      "[Parallel(n_jobs=-1)]: Done 572348 tasks      | elapsed: 29.1min\n",
      "[Parallel(n_jobs=-1)]: Done 574304 tasks      | elapsed: 29.2min\n",
      "[Parallel(n_jobs=-1)]: Done 576260 tasks      | elapsed: 29.3min\n",
      "[Parallel(n_jobs=-1)]: Done 578224 tasks      | elapsed: 29.4min\n",
      "[Parallel(n_jobs=-1)]: Done 580188 tasks      | elapsed: 29.5min\n",
      "[Parallel(n_jobs=-1)]: Done 582160 tasks      | elapsed: 29.6min\n",
      "[Parallel(n_jobs=-1)]: Done 584132 tasks      | elapsed: 29.7min\n",
      "[Parallel(n_jobs=-1)]: Done 586112 tasks      | elapsed: 29.8min\n",
      "[Parallel(n_jobs=-1)]: Done 588092 tasks      | elapsed: 29.9min\n",
      "[Parallel(n_jobs=-1)]: Done 590080 tasks      | elapsed: 30.0min\n",
      "[Parallel(n_jobs=-1)]: Done 592068 tasks      | elapsed: 30.1min\n",
      "[Parallel(n_jobs=-1)]: Done 594064 tasks      | elapsed: 30.2min\n",
      "[Parallel(n_jobs=-1)]: Done 596060 tasks      | elapsed: 30.3min\n",
      "[Parallel(n_jobs=-1)]: Done 598064 tasks      | elapsed: 30.4min\n",
      "[Parallel(n_jobs=-1)]: Done 600068 tasks      | elapsed: 30.5min\n",
      "[Parallel(n_jobs=-1)]: Done 602080 tasks      | elapsed: 30.7min\n",
      "[Parallel(n_jobs=-1)]: Done 604092 tasks      | elapsed: 30.7min\n",
      "[Parallel(n_jobs=-1)]: Done 606112 tasks      | elapsed: 30.8min\n",
      "[Parallel(n_jobs=-1)]: Done 608132 tasks      | elapsed: 30.9min\n",
      "[Parallel(n_jobs=-1)]: Done 610160 tasks      | elapsed: 31.0min\n",
      "[Parallel(n_jobs=-1)]: Done 612188 tasks      | elapsed: 31.1min\n",
      "[Parallel(n_jobs=-1)]: Done 614224 tasks      | elapsed: 31.2min\n",
      "[Parallel(n_jobs=-1)]: Done 616260 tasks      | elapsed: 31.3min\n",
      "[Parallel(n_jobs=-1)]: Done 618304 tasks      | elapsed: 31.4min\n",
      "[Parallel(n_jobs=-1)]: Done 620348 tasks      | elapsed: 31.5min\n",
      "[Parallel(n_jobs=-1)]: Done 622400 tasks      | elapsed: 31.6min\n",
      "[Parallel(n_jobs=-1)]: Done 624452 tasks      | elapsed: 31.7min\n",
      "[Parallel(n_jobs=-1)]: Done 626512 tasks      | elapsed: 31.8min\n",
      "[Parallel(n_jobs=-1)]: Done 628572 tasks      | elapsed: 31.9min\n",
      "[Parallel(n_jobs=-1)]: Done 630640 tasks      | elapsed: 32.0min\n",
      "[Parallel(n_jobs=-1)]: Done 632708 tasks      | elapsed: 32.1min\n",
      "[Parallel(n_jobs=-1)]: Done 634784 tasks      | elapsed: 32.2min\n",
      "[Parallel(n_jobs=-1)]: Done 636860 tasks      | elapsed: 32.3min\n",
      "[Parallel(n_jobs=-1)]: Done 638944 tasks      | elapsed: 32.4min\n",
      "[Parallel(n_jobs=-1)]: Done 641028 tasks      | elapsed: 32.5min\n",
      "[Parallel(n_jobs=-1)]: Done 643120 tasks      | elapsed: 32.6min\n",
      "[Parallel(n_jobs=-1)]: Done 645212 tasks      | elapsed: 32.7min\n",
      "[Parallel(n_jobs=-1)]: Done 647312 tasks      | elapsed: 32.8min\n",
      "[Parallel(n_jobs=-1)]: Done 649412 tasks      | elapsed: 32.9min\n",
      "[Parallel(n_jobs=-1)]: Done 651520 tasks      | elapsed: 33.0min\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1951s.) Setting batch_size=8.\n",
      "[Parallel(n_jobs=-1)]: Done 654716 tasks      | elapsed: 33.1min\n",
      "[Parallel(n_jobs=-1)]: Done 658948 tasks      | elapsed: 33.3min\n",
      "[Parallel(n_jobs=-1)]: Done 663180 tasks      | elapsed: 33.5min\n",
      "[Parallel(n_jobs=-1)]: Done 667428 tasks      | elapsed: 33.7min\n",
      "[Parallel(n_jobs=-1)]: Done 671676 tasks      | elapsed: 33.9min\n",
      "[Parallel(n_jobs=-1)]: Done 675940 tasks      | elapsed: 34.0min\n",
      "[Parallel(n_jobs=-1)]: Done 680204 tasks      | elapsed: 34.2min\n",
      "[Parallel(n_jobs=-1)]: Done 684484 tasks      | elapsed: 34.4min\n",
      "[Parallel(n_jobs=-1)]: Done 688764 tasks      | elapsed: 34.6min\n",
      "[Parallel(n_jobs=-1)]: Done 693060 tasks      | elapsed: 34.8min\n",
      "[Parallel(n_jobs=-1)]: Done 697356 tasks      | elapsed: 35.0min\n",
      "[Parallel(n_jobs=-1)]: Done 701668 tasks      | elapsed: 35.2min\n",
      "[Parallel(n_jobs=-1)]: Done 705980 tasks      | elapsed: 35.3min\n",
      "[Parallel(n_jobs=-1)]: Done 710308 tasks      | elapsed: 35.5min\n",
      "[Parallel(n_jobs=-1)]: Done 714636 tasks      | elapsed: 35.7min\n",
      "[Parallel(n_jobs=-1)]: Done 718980 tasks      | elapsed: 35.9min\n",
      "[Parallel(n_jobs=-1)]: Done 723324 tasks      | elapsed: 36.1min\n",
      "[Parallel(n_jobs=-1)]: Done 727684 tasks      | elapsed: 36.3min\n",
      "[Parallel(n_jobs=-1)]: Done 732044 tasks      | elapsed: 36.5min\n",
      "[Parallel(n_jobs=-1)]: Done 736420 tasks      | elapsed: 36.7min\n",
      "[Parallel(n_jobs=-1)]: Done 740796 tasks      | elapsed: 36.8min\n",
      "[Parallel(n_jobs=-1)]: Done 745188 tasks      | elapsed: 37.0min\n",
      "[Parallel(n_jobs=-1)]: Done 749580 tasks      | elapsed: 37.2min\n",
      "[Parallel(n_jobs=-1)]: Done 753988 tasks      | elapsed: 37.4min\n",
      "[Parallel(n_jobs=-1)]: Done 758396 tasks      | elapsed: 37.6min\n",
      "[Parallel(n_jobs=-1)]: Done 762820 tasks      | elapsed: 37.8min\n",
      "[Parallel(n_jobs=-1)]: Done 767244 tasks      | elapsed: 38.0min\n",
      "[Parallel(n_jobs=-1)]: Done 771684 tasks      | elapsed: 38.2min\n",
      "[Parallel(n_jobs=-1)]: Done 776124 tasks      | elapsed: 38.4min\n",
      "[Parallel(n_jobs=-1)]: Done 780580 tasks      | elapsed: 38.6min\n",
      "[Parallel(n_jobs=-1)]: Done 785036 tasks      | elapsed: 38.7min\n",
      "[Parallel(n_jobs=-1)]: Done 789508 tasks      | elapsed: 38.9min\n",
      "[Parallel(n_jobs=-1)]: Done 793980 tasks      | elapsed: 39.1min\n",
      "[Parallel(n_jobs=-1)]: Done 798468 tasks      | elapsed: 39.3min\n",
      "[Parallel(n_jobs=-1)]: Done 802956 tasks      | elapsed: 39.5min\n",
      "[Parallel(n_jobs=-1)]: Done 807460 tasks      | elapsed: 39.7min\n",
      "[Parallel(n_jobs=-1)]: Done 811964 tasks      | elapsed: 39.9min\n",
      "[Parallel(n_jobs=-1)]: Done 816484 tasks      | elapsed: 40.1min\n",
      "[Parallel(n_jobs=-1)]: Done 821004 tasks      | elapsed: 40.3min\n",
      "[Parallel(n_jobs=-1)]: Done 825540 tasks      | elapsed: 40.5min\n",
      "[Parallel(n_jobs=-1)]: Done 830076 tasks      | elapsed: 40.7min\n",
      "[Parallel(n_jobs=-1)]: Done 834628 tasks      | elapsed: 40.9min\n",
      "[Parallel(n_jobs=-1)]: Done 839180 tasks      | elapsed: 41.1min\n",
      "[Parallel(n_jobs=-1)]: Done 843748 tasks      | elapsed: 41.3min\n",
      "[Parallel(n_jobs=-1)]: Done 848316 tasks      | elapsed: 41.5min\n",
      "[Parallel(n_jobs=-1)]: Done 852900 tasks      | elapsed: 41.8min\n",
      "[Parallel(n_jobs=-1)]: Done 857484 tasks      | elapsed: 42.0min\n",
      "[Parallel(n_jobs=-1)]: Done 862084 tasks      | elapsed: 42.2min\n",
      "[Parallel(n_jobs=-1)]: Done 866684 tasks      | elapsed: 42.3min\n",
      "[Parallel(n_jobs=-1)]: Done 871300 tasks      | elapsed: 42.6min\n",
      "[Parallel(n_jobs=-1)]: Done 875916 tasks      | elapsed: 42.8min\n",
      "[Parallel(n_jobs=-1)]: Done 880548 tasks      | elapsed: 43.0min\n",
      "[Parallel(n_jobs=-1)]: Done 885180 tasks      | elapsed: 43.2min\n",
      "[Parallel(n_jobs=-1)]: Done 889828 tasks      | elapsed: 43.4min\n",
      "[Parallel(n_jobs=-1)]: Done 894476 tasks      | elapsed: 43.6min\n",
      "[Parallel(n_jobs=-1)]: Done 899140 tasks      | elapsed: 43.8min\n",
      "[Parallel(n_jobs=-1)]: Done 903804 tasks      | elapsed: 44.0min\n",
      "[Parallel(n_jobs=-1)]: Done 908484 tasks      | elapsed: 44.2min\n",
      "[Parallel(n_jobs=-1)]: Done 913164 tasks      | elapsed: 44.4min\n",
      "[Parallel(n_jobs=-1)]: Done 917860 tasks      | elapsed: 44.6min\n",
      "[Parallel(n_jobs=-1)]: Done 922556 tasks      | elapsed: 44.9min\n",
      "[Parallel(n_jobs=-1)]: Done 927268 tasks      | elapsed: 45.1min\n",
      "[Parallel(n_jobs=-1)]: Done 931980 tasks      | elapsed: 45.3min\n",
      "[Parallel(n_jobs=-1)]: Done 936708 tasks      | elapsed: 45.5min\n",
      "[Parallel(n_jobs=-1)]: Done 941436 tasks      | elapsed: 45.7min\n",
      "[Parallel(n_jobs=-1)]: Done 946180 tasks      | elapsed: 45.9min\n",
      "[Parallel(n_jobs=-1)]: Done 950924 tasks      | elapsed: 46.1min\n",
      "[Parallel(n_jobs=-1)]: Done 955684 tasks      | elapsed: 46.4min\n",
      "[Parallel(n_jobs=-1)]: Done 960444 tasks      | elapsed: 46.6min\n",
      "[Parallel(n_jobs=-1)]: Done 965220 tasks      | elapsed: 46.8min\n",
      "[Parallel(n_jobs=-1)]: Done 969996 tasks      | elapsed: 47.0min\n",
      "[Parallel(n_jobs=-1)]: Done 974788 tasks      | elapsed: 47.2min\n",
      "[Parallel(n_jobs=-1)]: Done 979580 tasks      | elapsed: 47.4min\n",
      "[Parallel(n_jobs=-1)]: Done 984388 tasks      | elapsed: 47.7min\n",
      "[Parallel(n_jobs=-1)]: Done 989196 tasks      | elapsed: 47.9min\n",
      "[Parallel(n_jobs=-1)]: Done 994020 tasks      | elapsed: 48.1min\n",
      "[Parallel(n_jobs=-1)]: Done 998844 tasks      | elapsed: 48.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1003684 tasks      | elapsed: 48.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1008524 tasks      | elapsed: 48.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1013380 tasks      | elapsed: 49.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1018236 tasks      | elapsed: 49.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1023108 tasks      | elapsed: 49.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1027980 tasks      | elapsed: 49.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1032868 tasks      | elapsed: 49.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1037756 tasks      | elapsed: 50.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1042660 tasks      | elapsed: 50.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1047564 tasks      | elapsed: 50.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1052484 tasks      | elapsed: 50.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1057404 tasks      | elapsed: 51.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1062340 tasks      | elapsed: 51.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1067276 tasks      | elapsed: 51.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1072228 tasks      | elapsed: 51.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1077180 tasks      | elapsed: 51.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1082148 tasks      | elapsed: 52.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1087116 tasks      | elapsed: 52.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1092100 tasks      | elapsed: 52.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1097084 tasks      | elapsed: 52.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1102084 tasks      | elapsed: 53.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1107084 tasks      | elapsed: 53.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1112100 tasks      | elapsed: 53.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1117116 tasks      | elapsed: 53.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1122148 tasks      | elapsed: 54.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1127180 tasks      | elapsed: 54.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1132228 tasks      | elapsed: 54.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1137276 tasks      | elapsed: 54.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1142340 tasks      | elapsed: 54.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1147404 tasks      | elapsed: 55.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1152484 tasks      | elapsed: 55.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1157564 tasks      | elapsed: 55.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1162660 tasks      | elapsed: 55.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1167756 tasks      | elapsed: 56.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1172868 tasks      | elapsed: 56.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1177980 tasks      | elapsed: 56.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1183108 tasks      | elapsed: 56.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1188236 tasks      | elapsed: 57.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1193380 tasks      | elapsed: 57.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1198524 tasks      | elapsed: 57.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1203684 tasks      | elapsed: 57.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1208844 tasks      | elapsed: 57.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1214020 tasks      | elapsed: 58.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1219196 tasks      | elapsed: 58.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1224388 tasks      | elapsed: 58.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1229580 tasks      | elapsed: 58.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1234788 tasks      | elapsed: 59.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1239996 tasks      | elapsed: 59.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1245220 tasks      | elapsed: 59.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1250444 tasks      | elapsed: 59.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1255684 tasks      | elapsed: 60.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1260924 tasks      | elapsed: 60.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1266180 tasks      | elapsed: 60.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1271436 tasks      | elapsed: 60.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1276708 tasks      | elapsed: 61.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1281980 tasks      | elapsed: 61.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1287268 tasks      | elapsed: 61.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1292556 tasks      | elapsed: 61.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1297860 tasks      | elapsed: 62.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1303164 tasks      | elapsed: 62.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1308484 tasks      | elapsed: 62.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1313804 tasks      | elapsed: 62.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1319140 tasks      | elapsed: 63.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1324476 tasks      | elapsed: 63.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1329828 tasks      | elapsed: 63.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1335180 tasks      | elapsed: 63.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1340548 tasks      | elapsed: 64.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1345916 tasks      | elapsed: 64.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1351300 tasks      | elapsed: 64.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1356684 tasks      | elapsed: 65.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1362084 tasks      | elapsed: 65.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1367484 tasks      | elapsed: 65.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1370872 out of 1370872 | elapsed: 65.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1370872"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def filter_img(x: str) -> Union[str, None]:\n",
    "    try:\n",
    "        image = Image.open(x)\n",
    "        processor(images=image, return_tensors=\"pt\")\n",
    "        # value = read_image(x, mode=ImageReadMode.RGB)\n",
    "        # if len(value.shape) ==3:\n",
    "        return x\n",
    "\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return None\n",
    "\n",
    "# read_image(all_image_path_list[0], mode=ImageReadMode.RGB).shape\n",
    "# clean_image_path_list = [filter_img(i) for i in tqdm(all_image_path_list[:100])]\n",
    "clean_image_path_list = Parallel(n_jobs=-1, verbose=10)(delayed(filter_img)(i) for i in all_image_path_list)\n",
    "len(clean_image_path_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1370013"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_image_path_list = [i for i in clean_image_path_list if i is not None]\n",
    "len(clean_image_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"bigdata/temp_dir\", exist_ok=True)\n",
    "pd.DataFrame({'image_path':clean_image_path_list}).to_csv(\"bigdata/temp_dir/temp_image_url.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mynet2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "110bc624a448454d574a0cd6cc76359fd86f75739e493913b3d71c2e04f2ffdb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
